import re
import util
from scrapy import signals
from scrapy.http import Response, HtmlResponse

class TencentDownloadMiddleware(object):
    pattern = re.compile(r"[\r\n\t]")

    def process_request(self, request, spider):
        #添加refer
        request.headers.setdefault(b'referer', 'https://news.qq.com')
        #添加头
        request.headers[b"User-Agent"] = util.getRandomUserAgent()

    def process_response(self, request, response, spider):
        temp = response.text
        temp = re.sub(self.pattern, "", temp)
        temp = temp.replace(u"\u3000", "").replace(u"\xa0", "")
        response = response.replace(body=temp)
        response._text = temp
        return response

class TencentMiddleware(object):

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)  